# GNU R script to analyze the results of the study, and generate
# tables.
#
# v0.1  12 Oct 2007
#
# Author: Ramon Casero <ramon.casero@oucs.ox.ac.uk>
# Copyright © 2007 University of Oxford
#
# This file is part of mta-tracker.
#
# mta-tracker is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# mta-tracker is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with mta-tracker.  If not, see <http://www.gnu.org/licenses/>.
#
# ChangeLog
#
# * 2008-08-21 (rcasero):
#
#   ** Upgrade licence to GPL v3.
#
# * 2007-10-12 (rcasero): v0.1.
#
#   - First working release.

# read file with merged results
data <- read.table("log-total-R.txt", sep="\t", header=TRUE)

# attach table so that we can refer to the columns directly
attach(data)

###########################################################
# FUNCTIONS
###########################################################
# format for percentages, e.g. 0.90543 becomes (90.5%)
pc <- function(x){
  x <- formatC(x * 100,dig=1,format="f")
  x[x!="NA"] <- gsub("(.*)","(\\1%)",x[x!="NA"],perl=T)
  x[x=="NA"] <- gsub("(.*)","(\\1)",x[x=="NA"],perl=T)
  x
}
# number of different elements, i.e. number of levels of a factor
nl <- function(x){
  nlevels(factor(x))
}

###########################################################
# DOMAIN TABLE
###########################################################

# total number of domains
dom.tot <- nlevels(domain)

# number of non-registered domains
insti.reg.mask <- insti != "not registered"
dom.nreg.tot <- nl(domain[!insti.reg.mask])

# number of registered domains...
dom.reg.tot <- nl(domain[insti.reg.mask])

# ... in FE/HE/Other
dom.reg.fe <- nl(domain[insti.reg.mask & (type == "fe")])
dom.reg.he <- nl(domain[insti.reg.mask & (type == "he")])
dom.reg.oth <- nl(domain[insti.reg.mask & (type == "none")])
dom.reg.unk <- nl(domain[insti.reg.mask & (type == "unknown")])

# sanity checks
if ((dom.reg.tot != dom.reg.fe + dom.reg.he + dom.reg.oth +
     dom.reg.unk) | (dom.reg.unk != 0)) {
  stop("Sanity check: domain table... failed")
} else {
  print("Sanity check: domain table... OK")
}

# create columns for output table
table.dom.tot <- c(dom.tot, dom.nreg.tot, dom.reg.tot)
table.dom.fe <- c(NA, NA, dom.reg.fe)
table.dom.he <- c(NA, NA, dom.reg.he)
table.dom.oth <- c(NA, NA, dom.reg.oth)

# create table
# Percentages are computed:
#  * for the Total column: with respect to (Total,Total)
#  * otherwise: with respect to the first element in the row
table.dom <-
  data.frame(
             table.dom.tot,
             pc(table.dom.tot / table.dom.tot[1]),
             table.dom.fe,
             pc(table.dom.fe / table.dom.tot),
             table.dom.he,
             pc(table.dom.he / table.dom.tot),
             table.dom.oth,
             pc(table.dom.oth / table.dom.tot),
                        row.names=c("Domains", "+ Not registered",
                          "+ Registered"))
names(table.dom) <- c("Total", "(%)", "FE", "(%)", "HE", "(%)", "Other", "(%)")

#table.dom

# write table to CSV file
# note: we could use write.table to produce a txt table, but it will
# look much better doing
# $ cat tables/table-domain.csv | sed 's/^,/ ,/' | column -ts ','
write("Top level domains in ac.uk", file="tables/table-domain.csv")
write.csv(table.dom, file="tables/table-domain.csv", quote=FALSE,
          append=T)

###########################################################
# INSTITUTION AND SERVER TABLE
###########################################################

# total number of institutions
insti.tot <- nl(insti[insti.reg.mask])

# ... in FE/HE/Other
insti.fe <- nl(insti[insti.reg.mask & (type == "fe")])
insti.he <- nl(insti[insti.reg.mask & (type == "he")])
insti.oth <- nl(insti[insti.reg.mask & (type == "none")])
insti.unk <- nl(insti[insti.reg.mask & (type == "unknown")])

# sanity checks:
if ((insti.tot != insti.fe + insti.he + insti.oth) |
    (insti.unk != 0)) {
  stop("Sanity check: institution/server table... failed")
} else {
  print("Sanity check: institution/server table... OK")
}

# list of servers for which we have a name
serv.name.mask <- (server != "host not found for domain") &
(server != "no MX record")
# ... but no IP address
serv.noip.mask <- mta=="server not found"
# ... with IP address
serv.ip.mask <- mta!="server not found" & mta != "no server"

# total number of servers
serv.name.tot <- nl(server[serv.name.mask])
serv.ip.tot <- nl(server[serv.ip.mask])
serv.noip.tot <- nl(server[serv.noip.mask])

# ... in FE/HE/Other
serv.name.fe <- nl(server[serv.name.mask & (type == "fe")])
serv.name.he <- nl(server[serv.name.mask & (type == "he")])
serv.name.oth <- nl(server[serv.name.mask & (type == "none")]) +
  nl(server[serv.name.mask & (type == "unknown")])
serv.ip.fe <- nl(server[serv.ip.mask & (type == "fe")])
serv.ip.he <- nl(server[serv.ip.mask & (type == "he")])
serv.ip.oth <- nl(server[serv.ip.mask & (type == "none")]) +
  nl(server[serv.ip.mask & (type == "unknown")])

# sanity checks:
if ( (serv.name.tot != serv.noip.tot + serv.ip.tot)
    | (serv.name.tot > serv.name.fe + serv.name.he + serv.name.oth)) {
  stop("Sanity check: institution/server table... failed")
} else {
  print("Sanity check: institution/server table... OK")
}
if (serv.ip.tot > serv.ip.fe + serv.ip.he + serv.ip.oth) {
  stop("Sanity check: institution/server table... failed")
} else {
  print("Sanity check: institution/server table... OK")
}

# create columns for output table
table.instiserv.tot <- c(insti.tot, serv.name.tot, serv.ip.tot)
table.instiserv.fe <- c(insti.fe, serv.name.fe, serv.ip.fe)
table.instiserv.he <- c(insti.he, serv.name.he, serv.ip.he)
table.instiserv.oth <- c(insti.oth, serv.name.oth, serv.ip.oth)

# create table
table.instiserv <-
  data.frame(
             table.instiserv.tot,
             pc(table.instiserv.tot / table.instiserv.tot),
             table.instiserv.fe,
             pc(table.instiserv.fe / table.instiserv.tot),
             table.instiserv.he,
             pc(table.instiserv.he / table.instiserv.tot),
             table.instiserv.oth,
             pc(table.instiserv.oth / table.instiserv.tot),
                        row.names=c("Institutions", "Server names",
                          "+ With IP address"))
names(table.instiserv) <- c("Total", "(%)", "FE", "(%)", "HE", "(%)",
                            "Other", "(%)")

#table.instiserv

# write table to CSV file
write("Institutions in ac.uk and servers used by them",
      file="tables/table-instiserv.csv")
write.csv(table.instiserv, file="tables/table-instiserv.csv",
          quote=FALSE, append=TRUE)

###########################################################
# EMAIL TABLE
###########################################################

# institutions with email (total and in house)
insti.email.mask <- insti.reg.mask & serv.ip.mask
insti.email.in.mask <- insti.email.mask & (prov == "in house")
insti.email.unk.mask <- insti.email.mask & (prov == "unknown provider")

insti.email.tot <- nl(insti[insti.email.mask])

# ... in FE/HE/Other
insti.email.fe <- nl(insti[insti.email.mask & type == "fe"])
insti.email.he <- nl(insti[insti.email.mask & type == "he"])
insti.email.oth <- nl(insti[insti.email.mask & type == "none"])
insti.email.unk <- nl(insti[insti.email.mask & type == "unknown"])

# in house
insti.email.in.tot <- nl(insti[insti.email.in.mask])
insti.email.in.fe <- nl(insti[insti.email.in.mask & type == "fe"])
insti.email.in.he <- nl(insti[insti.email.in.mask & type == "he"])
insti.email.in.oth <- nl(insti[insti.email.in.mask
                               & (type == "none" | type == "unknown")])

# unknown provider
insti.email.unk.tot <- nl(insti[insti.email.unk.mask])
insti.email.unk.fe <- nl(insti[insti.email.unk.mask & type == "fe"])
insti.email.unk.he <- nl(insti[insti.email.unk.mask & type == "he"])
insti.email.unk.oth <- nl(insti[insti.email.unk.mask
                               & (type == "none" | type == "unknown")])

# outsourced
insti.email.out.mask <- insti.email.mask & (prov != "in house")
insti.email.out.tot <- nl(insti[insti.email.out.mask])
insti.email.out.fe <- nl(insti[insti.email.out.mask & type == "fe"])
insti.email.out.he <- nl(insti[insti.email.out.mask & type == "he"])
insti.email.out.oth <- nl(insti[insti.email.out.mask & type == "none"])
insti.email.out.unk <- nl(insti[insti.email.out.mask & type == "unknown"])

# overlap
insti.email.ove.tot <- insti.email.out.tot + insti.email.in.tot -
  insti.email.tot - insti.email.unk.tot
insti.email.ove.fe <- insti.email.out.fe + insti.email.in.fe -
  insti.email.fe - insti.email.unk.fe
insti.email.ove.he <- insti.email.out.he + insti.email.in.he -
  insti.email.he - insti.email.unk.he
insti.email.ove.oth <- insti.email.out.oth + insti.email.in.oth -
  insti.email.oth - insti.email.unk.oth

# sanity checks:
if ((insti.email.tot != insti.email.fe + insti.email.he + insti.email.oth)
    | (insti.email.in.tot != insti.email.in.fe + insti.email.in.he
       + insti.email.in.oth)
    | (insti.email.out.tot != insti.email.out.fe + insti.email.out.he
       + insti.email.out.oth)
    ) {
  stop("Sanity check: institutions with email table... failed")
} else {
  print("Sanity check: institutions with email table... OK")
}

# create columns for output table
table.insti.email.tot <- c(insti.tot, insti.email.tot, insti.email.unk.tot,
                     insti.email.in.tot, insti.email.out.tot,
                           insti.email.ove.tot)
table.insti.email.fe <- c(insti.fe, insti.email.fe, insti.email.unk.fe,
                    insti.email.in.fe, insti.email.out.fe,
                          insti.email.ove.fe)
table.insti.email.he <- c(insti.he, insti.email.he, insti.email.unk.he,
                    insti.email.in.he, insti.email.out.he,
                          insti.email.ove.he)
table.insti.email.oth <- c(insti.oth, insti.email.oth, insti.email.unk.oth,
                     insti.email.in.oth, insti.email.out.oth,
                           insti.email.ove.oth)

# create table
table.insti.email <-
  data.frame(
             table.insti.email.tot,
             pc(table.insti.email.tot / table.insti.email.tot[1]),
             table.insti.email.fe,
             pc(table.insti.email.fe / table.insti.email.fe[1]),
             table.insti.email.he,
             pc(table.insti.email.he / table.insti.email.he[1]),
             table.insti.email.oth,
             pc(table.insti.email.oth / table.insti.email.oth[1]),
                        row.names=c("Institutions", "+ With email",
                          "++ Unknown provider", "++ In house",
                          "++ Outsourced", "++ Overlap"))
names(table.insti.email) <- c("Total", "(%)", "FE", "(%)", "HE", "(%)",
                        "Other", "(%)")

#table.insti.email

# write table to CSV file
write("Email by institution in ac.uk", file="tables/table-insti-email.csv")
write.csv(table.insti.email, file="tables/table-insti-email.csv",
          quote=FALSE, append=TRUE)

###########################################################
# OUTSOURCED EMAIL BY INSTITUTION TABLE
###########################################################

# outsourced to ac.uk
prov.acuk.out.mask <- array(FALSE, length(insti))
prov.acuk.out.mask[grep(".*ac\.uk$",prov)] <- TRUE

insti.email.out.acuk.tot <- nl(insti[prov.acuk.out.mask])

# ... for FE, HE and Others
insti.email.out.acuk.fe <- nl(insti[prov.acuk.out.mask
                                    & type == "fe"])
insti.email.out.acuk.he <- nl(insti[prov.acuk.out.mask
                                    & type == "he"])
insti.email.out.acuk.oth <- nl(insti[prov.acuk.out.mask
                                     & (type == "none" | type == "unknown")])

# outsourced to non-ac.uk
prov.nacuk.out.mask <- !(prov.acuk.out.mask |
                         (prov=="no provider") |
                         (prov=="in house") |
                         (prov=="unknown provider"))

insti.email.out.nacuk.tot <- nl(insti[prov.nacuk.out.mask])

# ... for FE, HE and Others
insti.email.out.nacuk.fe <- nl(insti[prov.nacuk.out.mask
                                    & type == "fe"])
insti.email.out.nacuk.he <- nl(insti[prov.nacuk.out.mask
                                    & type == "he"])
insti.email.out.nacuk.oth <- nl(insti[prov.nacuk.out.mask
                                      & (type == "none" | type == "unknown")])

# overlap
insti.email.out.ove.tot <- insti.email.out.nacuk.tot +
  insti.email.out.acuk.tot - insti.email.out.tot
insti.email.out.ove.fe <- insti.email.out.nacuk.fe +
  insti.email.out.acuk.fe - insti.email.out.fe
insti.email.out.ove.he <- insti.email.out.nacuk.he +
  insti.email.out.acuk.he - insti.email.out.he
insti.email.out.ove.oth <- insti.email.out.nacuk.oth +
  insti.email.out.acuk.oth - insti.email.out.oth

# create columns for output table
table.insti.email.out.tot <-
  c(insti.email.out.tot, insti.email.out.acuk.tot,
    insti.email.out.nacuk.tot, insti.email.out.ove.tot)
table.insti.email.out.fe <-
  c(insti.email.out.fe, insti.email.out.acuk.fe,
    insti.email.out.nacuk.fe, insti.email.out.ove.fe)
table.insti.email.out.he <-
  c(insti.email.out.he, insti.email.out.acuk.he,
    insti.email.out.nacuk.he, insti.email.out.ove.he)
table.insti.email.out.oth <-
  c(insti.email.out.oth, insti.email.out.acuk.oth,
    insti.email.out.nacuk.oth, insti.email.out.ove.oth)

# create table
table.insti.email.out <-
  data.frame(
             table.insti.email.out.tot,
             pc(table.insti.email.out.tot / table.insti.email.out.tot[1]),
             table.insti.email.out.fe,
             pc(table.insti.email.out.fe / table.insti.email.out.fe[1]),
             table.insti.email.out.he,
             pc(table.insti.email.out.he / table.insti.email.out.he[1]),
             table.insti.email.out.oth,
             pc(table.insti.email.out.oth / table.insti.email.out.oth[1]),
                        row.names=c("++ Outsourced", "+++ To ac.uk",
                          "+++ To non-ac.uk", "+++ Overlap"))
names(table.insti.email.out) <- c("Total", "(%)", "FE", "(%)", "HE", "(%)",
                                  "Other", "(%)")

#table.insti.email.out

# write table to CSV file
write("Outsourced email by institution in ac.uk",
      file="tables/table-insti-email-out.csv")
write.csv(table.insti.email.out,
          file="tables/table-insti-email-out.csv", quote=FALSE,
          append=TRUE)

###########################################################
# EMAIL BY SERVER TABLE
###########################################################

# servers with IP address in ac.uk domain
serv.ip.acuk.mask <- serv.ip.mask & (prov.acuk.out.mask | prov=="in house")
serv.ip.acuk.tot <- nl(server[serv.ip.acuk.mask])

# servers with IP address outside ac.uk domain
serv.ip.nacuk.mask <- prov!="no provider" & prov!="in house" &
prov!="unknown provider" & mta!="server not found" &
!prov.acuk.out.mask

serv.ip.nacuk.tot <- nl(server[serv.ip.nacuk.mask])

# unknown provider
serv.ip.unk.mask <- serv.ip.mask & prov=="unknown provider"
serv.ip.unk.tot <- nl(server[serv.ip.unk.mask])

# sanity checks:
if (serv.ip.tot != serv.ip.acuk.tot + serv.ip.nacuk.tot +
    serv.ip.unk.tot)
{
  stop("Sanity check: outsourced email by server table... failed")
} else {
  print("Sanity check: outsourced email by server table... OK")
}

# ... in FE/HE/Other
serv.ip.acuk.fe <- nl(server[serv.ip.acuk.mask & (type == "fe")])
serv.ip.acuk.he <- nl(server[serv.ip.acuk.mask & (type == "he")])
serv.ip.acuk.oth <- nl(server[serv.ip.acuk.mask
                              & (type == "none" | type == "unknown")])
serv.ip.nacuk.fe <- nl(server[serv.ip.nacuk.mask & (type == "fe")])
serv.ip.nacuk.he <- nl(server[serv.ip.nacuk.mask & (type == "he")])
serv.ip.nacuk.oth <- nl(server[serv.ip.nacuk.mask
                              & (type == "none" | type == "unknown")])
serv.ip.unk.fe <- nl(server[serv.ip.unk.mask & (type == "fe")])
serv.ip.unk.he <- nl(server[serv.ip.unk.mask & (type == "he")])
serv.ip.unk.oth <- nl(server[serv.ip.unk.mask
                             & (type == "none" | type == "unknown")])

# create columns for output table
table.serv.tot <- c(serv.ip.tot, serv.ip.unk.tot, serv.ip.nacuk.tot,
                    serv.ip.acuk.tot)
table.serv.fe <- c(serv.ip.fe, serv.ip.unk.fe, serv.ip.nacuk.fe,
                   serv.ip.acuk.fe)
table.serv.he <- c(serv.ip.he, serv.ip.unk.he, serv.ip.nacuk.he,
                   serv.ip.acuk.he)
table.serv.oth <- c(serv.ip.oth, serv.ip.unk.oth, serv.ip.nacuk.oth,
                    serv.ip.acuk.oth)

# create table
table.serv.email <-
  data.frame(
             table.serv.tot,
             pc(table.serv.tot / table.serv.tot[1]),
             table.serv.fe,
             pc(table.serv.fe / table.serv.fe[1]),
             table.serv.he,
             pc(table.serv.he / table.serv.he[1]),
             table.serv.oth,
             pc(table.serv.oth / table.serv.oth[1]),
             row.names=c("Servers with valid IP", "+ Unknown provider",
               "+ Outside ac.uk", "+ In ac.uk"))
names(table.serv.email) <- c("Total", "(%)", "FE", "(%)", "HE", "(%)",
                       "Other", "(%)")

#table.serv.email

# write table to CSV file
write("Email by servers for ac.uk domains",
      file="tables/table-serv-email.csv")
write.csv(table.serv.email,
          file="tables/table-serv-email.csv", quote=FALSE,
          append=TRUE)

###########################################################
# AC.UK EMAIL BY SERVER TABLE
###########################################################

# in house servers with IP address in ac.uk domain
serv.ip.acuk.in.mask <- serv.ip.acuk.mask & prov=="in house"
serv.ip.acuk.in.tot <- nl(server[serv.ip.acuk.in.mask])

# servers for other institutions with IP address in ac.uk domain
serv.ip.acuk.out.mask <- serv.ip.acuk.mask & prov!="in house" &
prov!="unknown provider" & prov!="no provider"
serv.ip.acuk.out.tot <- nl(server[serv.ip.acuk.out.mask])

# overlap
serv.ip.acuk.ove.tot <- serv.ip.acuk.out.tot + serv.ip.acuk.in.tot -
  serv.ip.acuk.tot

# ... FE, HE and Other
serv.ip.acuk.in.fe <- nl(server[serv.ip.acuk.in.mask & type=="fe"])
serv.ip.acuk.in.he <- nl(server[serv.ip.acuk.in.mask & type=="he"])
serv.ip.acuk.in.oth <- nl(server[serv.ip.acuk.in.mask
                          & (type=="none" | type=="unknown")])

serv.ip.acuk.out.fe <- nl(server[serv.ip.acuk.out.mask & type=="fe"])
serv.ip.acuk.out.he <- nl(server[serv.ip.acuk.out.mask & type=="he"])
serv.ip.acuk.out.oth <- nl(server[serv.ip.acuk.out.mask
                          & (type=="none" | type=="unknown")])

serv.ip.acuk.ove.fe <- serv.ip.acuk.out.fe + serv.ip.acuk.in.fe -
  serv.ip.acuk.fe
serv.ip.acuk.ove.he <- serv.ip.acuk.out.he + serv.ip.acuk.in.he -
  serv.ip.acuk.he
serv.ip.acuk.ove.oth <- serv.ip.acuk.out.oth + serv.ip.acuk.in.oth -
  serv.ip.acuk.oth

# create columns for output table
table.serv.acuk.tot <- c(serv.ip.acuk.tot, serv.ip.acuk.in.tot,
                         serv.ip.acuk.out.tot, serv.ip.acuk.ove.tot)
table.serv.acuk.fe <- c(serv.ip.acuk.fe, serv.ip.acuk.in.fe,
                        serv.ip.acuk.out.fe, serv.ip.acuk.ove.fe)
table.serv.acuk.he <- c(serv.ip.acuk.he, serv.ip.acuk.in.he,
                        serv.ip.acuk.out.he, serv.ip.acuk.ove.he)
table.serv.acuk.oth <- c(serv.ip.acuk.oth, serv.ip.acuk.in.oth,
                         serv.ip.acuk.out.oth, serv.ip.acuk.ove.oth)

# create table
table.serv.email.acuk <-
  data.frame(
             table.serv.acuk.tot,
             pc(table.serv.acuk.tot / table.serv.acuk.tot[1]),
             table.serv.acuk.fe,
             pc(table.serv.acuk.fe / table.serv.acuk.fe[1]),
             table.serv.acuk.he,
             pc(table.serv.acuk.he / table.serv.acuk.he[1]),
             table.serv.acuk.oth,
             pc(table.serv.acuk.oth / table.serv.acuk.oth[1]),
             row.names=c("Servers in ac.uk", "+ For own institution",
               "+ For other institutions", "+ Overlap"))
names(table.serv.email.acuk) <- c("Total", "(%)", "FE", "(%)", "HE", "(%)",
                       "Other", "(%)")

#table.serv.email.acuk

# write table to CSV file
write("Email by ac.uk servers for ac.uk domains",
      file="tables/table-serv-email-acuk.csv")
write.csv(table.serv.email.acuk,
          file="tables/table-serv-email-acuk.csv", quote=FALSE,
          append=TRUE)

###########################################################
# EXTERNAL PROVIDERS BY INSTITUTION TABLE
###########################################################

# get list of external providers
prov.nacuk <- c(levels(factor(prov[prov.nacuk.out.mask])))

# number of providers
N <- length(prov.nacuk)

# vector of column names
prov.names <- as.list(array(0, N))

# create vectors for counting institutions
prov.insti.tot <- array(0, N)
prov.insti.fe <- array(0, N)
prov.insti.he <- array(0, N)
prov.insti.oth <- array(0, N)

i <- 1
for (p in prov.nacuk) {
  prov.insti.tot[i] <- nl(insti[prov==p]);
  prov.names[[i]] <- paste("Provider", i, sep=" ")
  i <- i+1
}

# sort indices of providers from most popular to least
idx <- sort(prov.insti.tot, index.return=TRUE, dec=TRUE)

# we use this to sort prov.nacuk, so that it won't be necessary to
# sort the results again later
prov.nacuk <- prov.nacuk[idx$ix]
prov.insti.tot <- idx$x

i <- 1
for (p in prov.nacuk) {
  prov.insti.fe[i] <- nl(insti[prov==p & type=="fe"]);
  prov.insti.he[i] <- nl(insti[prov==p & type=="he"]);
  prov.insti.oth[i] <- nl(insti[prov==p & (type=="none" | type=="unknown")]);
  i <- i+1
}

# create columns for output table
table.prov.insti.tot <- c(sum(prov.insti.tot), prov.insti.tot)
table.prov.insti.fe <- c(sum(prov.insti.fe), prov.insti.fe)
table.prov.insti.he <- c(sum(prov.insti.he), prov.insti.he)
table.prov.insti.oth <-c(sum(prov.insti.oth), prov.insti.oth)

# create table
table.prov.insti <-
  data.frame(
             table.prov.insti.tot,
             pc(table.prov.insti.tot / table.prov.insti.tot[1]),
             table.prov.insti.fe,
             pc(table.prov.insti.fe / table.prov.insti.fe[1]),
             table.prov.insti.he,
             pc(table.prov.insti.he / table.prov.insti.he[1]),
             table.prov.insti.oth,
             pc(table.prov.insti.oth / table.prov.insti.oth[1]),
             row.names=c("Total",prov.names))
names(table.prov.insti) <- c("Total", "(%)", "FE", "(%)", "HE", "(%)",
                       "Other", "(%)")

#table.prov.insti

# cumsum(sort(table.prov.insti.fe[2:248], decr=T))/table.prov.insti.fe[1]*100
# cumsum(sort(table.prov.insti.he[2:248], decr=T))/table.prov.insti.he[1]*100
# cumsum(sort(table.prov.insti.oth[2:248], decr=T))/table.prov.insti.oth[1]*100

# write table to CSV file
write("External email providers by institution",
      file="tables/table-prov-insti.csv")
write.csv(table.prov.insti,
          file="tables/table-prov-insti.csv", quote=FALSE,
          append=TRUE)

###########################################################
# EXTERNAL PROVIDERS BY SERVER TABLE
###########################################################

# create vectors for counting institutions
prov.serv.tot <- array(0, N)
prov.serv.fe <- array(0, N)
prov.serv.he <- array(0, N)
prov.serv.oth <- array(0, N)

i <- 1
for (p in prov.nacuk) {
  prov.serv.tot[i] <- nl(server[prov==p]);
  prov.names[[i]] <- paste("Provider", i, sep=" ")
  i <- i+1
}

# sort indices of providers from most popular to least
idx <- sort(prov.serv.tot, index.return=TRUE, dec=TRUE)

# we use this to sort prov.nacuk, so that it won't be necessary to
# sort the results again later
prov.nacuk <- prov.nacuk[idx$ix]
prov.serv.tot <- idx$x
prov.names <- prov.names[idx$ix]
  
i <- 1
for (p in prov.nacuk) {
  prov.serv.fe[i] <- nl(server[prov==p & type=="fe"]);
  prov.serv.he[i] <- nl(server[prov==p & type=="he"]);
  prov.serv.oth[i] <- nl(server[prov==p & (type=="none" | type=="unknown")]);
  i <- i+1
}

# create columns for output table
table.prov.serv.tot <- c(sum(prov.serv.tot),prov.serv.tot)
table.prov.serv.fe <- c(sum(prov.serv.fe),prov.serv.fe)
table.prov.serv.he <- c(sum(prov.serv.he),prov.serv.he)
table.prov.serv.oth <-c(sum(prov.serv.oth),prov.serv.oth)


# create table
table.prov.serv <-
  data.frame(
             table.prov.serv.tot,
             pc(table.prov.serv.tot / table.prov.serv.tot[1]),
             table.prov.serv.fe,
             pc(table.prov.serv.fe / table.prov.serv.fe[1]),
             table.prov.serv.he,
             pc(table.prov.serv.he / table.prov.serv.he[1]),
             table.prov.serv.oth,
             pc(table.prov.serv.oth / table.prov.serv.oth[1]),
             row.names=c("Total",prov.names))
names(table.prov.serv) <- c("Total", "(%)", "FE", "(%)", "HE", "(%)",
                       "Other", "(%)")

#table.prov.serv

# cumsum(sort(table.prov.serv.fe[2:248], decr=T))/table.prov.serv.fe[1]*100
# cumsum(sort(table.prov.serv.he[2:248], decr=T))/table.prov.serv.he[1]*100
# cumsum(sort(table.prov.serv.oth[2:248], decr=T))/table.prov.serv.oth[1]*100

# write table to CSV file
write("External email providers by server",
      file="tables/table-prov-serv.csv")
write.csv(table.prov.serv,
          file="tables/table-prov-serv.csv", quote=FALSE,
          append=TRUE)

###########################################################
# MTA BY AC.UK INSTITUTION TABLE
###########################################################

# load MTA data
data.mta <- read.table("mta-names.txt", sep=",")
names(data.mta) <- c("string", "tag", "systype", "homepage")
attach(data.mta)

# get list of mtas 
mta.acuk <- c(levels(factor(tag[systype=="mta"])))

# number of providers
N <- length(mta.acuk)

# total number of institutions that have either in house email, or
# outsource to ac.uk
mta.insti.inacuk.mask <- prov.acuk.out.mask | insti.email.in.mask
mta.insti.inacuk.tot <- nl(insti[mta.insti.inacuk.mask])
mta.insti.inacuk.fe <- nl(insti[mta.insti.inacuk.mask & type=="fe"])
mta.insti.inacuk.he <- nl(insti[mta.insti.inacuk.mask & type=="he"])
mta.insti.inacuk.oth <- nl(insti[mta.insti.inacuk.mask
                                 & (type=="none" | type=="unknown")])

# institutions with email but unknown MTAs
mta.insti.unkmta.tot <- nl(insti[mta=="unknown MTA" & mta.insti.inacuk.mask])
mta.insti.unkmta.fe <- nl(insti[mta=="unknown MTA"
                                & mta.insti.inacuk.mask & type=="fe"])
mta.insti.unkmta.he <- nl(insti[mta=="unknown MTA" & mta.insti.inacuk.mask
                                & type=="he"])
mta.insti.unkmta.oth <- nl(insti[mta=="unknown MTA"
                                 & mta.insti.inacuk.mask
                                 & (type=="none" | type=="unknown")])

# create vectors for counting institutions with known MTAs
mta.insti.tot <- array(0, N)
mta.insti.fe <- array(0, N)
mta.insti.he <- array(0, N)
mta.insti.oth <- array(0, N)

# known MTAs
i <- 1
for (p in mta.acuk) {
  mta.insti.tot[i] <- nl(insti[mta==p & mta.insti.inacuk.mask]);
  i <- i+1
}

# sort indices of providers from most popular to least
idx <- sort(mta.insti.tot, index.return=TRUE, dec=TRUE)

# we use this to sort prov.nacuk, so that it won't be necessary to
# sort the results again later
mta.acuk <- mta.acuk[idx$ix]
mta.insti.tot <- idx$x

i <- 1
for (p in mta.acuk) {
  mta.insti.fe[i] <- nl(insti[mta==p & mta.insti.inacuk.mask
                              & type=="fe"]);
  mta.insti.he[i] <- nl(insti[mta==p & mta.insti.inacuk.mask
                              & type=="he"]);
  mta.insti.oth[i] <- nl(insti[mta==p & mta.insti.inacuk.mask
                               & (type=="none" | type=="unknown")]);
  i <- i+1
}

# create columns for output table
table.mta.insti.tot <- c(mta.insti.inacuk.tot,
                           mta.insti.unkmta.tot,
                         sum(mta.insti.tot), mta.insti.tot)
table.mta.insti.fe <- c(mta.insti.inacuk.fe,
                        mta.insti.unkmta.fe,
                        sum(mta.insti.fe), mta.insti.fe)
table.mta.insti.he <- c(mta.insti.inacuk.he,
                        mta.insti.unkmta.he,
                        sum(mta.insti.he), mta.insti.he)
table.mta.insti.oth <-c(mta.insti.inacuk.oth,
                        mta.insti.unkmta.oth,
                        sum(mta.insti.oth), mta.insti.oth)

# create table
table.mta.insti <-
  data.frame(
             table.mta.insti.tot,
             pc(table.mta.insti.tot / table.mta.insti.tot[1]),
             table.mta.insti.fe,
             pc(table.mta.insti.fe / table.mta.insti.fe[1]),
             table.mta.insti.he,
             pc(table.mta.insti.he / table.mta.insti.he[1]),
             table.mta.insti.oth,
             pc(table.mta.insti.oth / table.mta.insti.oth[1]),
             row.names=c("Institutions in ac.uk with email",
               "Running unknown MTAs",
               "Running known MTAs", mta.acuk))
names(table.mta.insti) <- c("Total", "(%)", "FE", "(%)", "HE", "(%)",
                       "Other", "(%)")

#table.mta.insti

# write table to CSV file
write("Mail Transfer Agents (MTAs) by institution",
      file="tables/table-mta-insti.csv")
write.csv(table.mta.insti,
          file="tables/table-mta-insti.csv", quote=FALSE,
          append=TRUE)

###########################################################
# FILTERS BY AC.UK INSTITUTION TABLE
###########################################################

# get list of mtas 
filter.acuk <- c(levels(factor(tag[systype=="filter"])))

# number of providers
N <- length(filter.acuk)

# create vectors for counting institutions
filter.insti.tot <- array(0, N)
filter.insti.fe <- array(0, N)
filter.insti.he <- array(0, N)
filter.insti.oth <- array(0, N)

i <- 1
for (p in filter.acuk) {
  filter.insti.tot[i] <- nl(insti[mta==p & mta.insti.inacuk.mask]);
  i <- i+1
}

# sort indices of providers from most popular to least
idx <- sort(filter.insti.tot, index.return=TRUE, dec=TRUE)

# we use this to sort prov.nacuk, so that it won't be necessary to
# sort the results again later
filter.acuk <- filter.acuk[idx$ix]
filter.insti.tot <- idx$x

i <- 1
for (p in filter.acuk) {
  filter.insti.fe[i] <- nl(insti[mta==p & mta.insti.inacuk.mask
                                 & type=="fe"]);
  filter.insti.he[i] <- nl(insti[mta==p & mta.insti.inacuk.mask
                                 & type=="he"]);
  filter.insti.oth[i] <- nl(insti[mta==p & mta.insti.inacuk.mask
                                  & (type=="none" | type=="unknown")]);
  i <- i+1
}

# create columns for output table
table.filter.insti.tot <- c(insti.email.tot,
                         sum(filter.insti.tot), filter.insti.tot)
table.filter.insti.fe <- c(insti.email.fe,
                        sum(filter.insti.fe), filter.insti.fe)
table.filter.insti.he <- c(insti.email.he,
                        sum(filter.insti.he), filter.insti.he)
table.filter.insti.oth <-c(insti.email.oth,
                        sum(filter.insti.oth), filter.insti.oth)

# create table
table.filter.insti <-
  data.frame(
             table.filter.insti.tot,
             pc(table.filter.insti.tot / table.filter.insti.tot[1]),
             table.filter.insti.fe,
             pc(table.filter.insti.fe / table.filter.insti.fe[1]),
             table.filter.insti.he,
             pc(table.filter.insti.he / table.filter.insti.he[1]),
             table.filter.insti.oth,
             pc(table.filter.insti.oth / table.filter.insti.oth[1]),
             row.names=c("Institutions in ac.uk with email",
               "Total filters", filter.acuk))
names(table.filter.insti) <- c("Total", "(%)", "FE", "(%)", "HE", "(%)",
                       "Other", "(%)")

#table.filter.insti

# write table to CSV file
write("Filters by institution",
      file="tables/table-filter-insti.csv")
write.csv(table.filter.insti,
          file="tables/table-filter-insti.csv", quote=FALSE,
          append=TRUE)

###########################################################
# MTA BY AC.UK SERVER TABLE
###########################################################

# number of providers
N <- length(mta.acuk)

# create vectors for counting institutions
mta.serv.tot <- array(0, N)
mta.serv.fe <- array(0, N)
mta.serv.he <- array(0, N)
mta.serv.oth <- array(0, N)

i <- 1
for (p in mta.acuk) {
  mta.serv.tot[i] <- nl(server[mta==p & serv.ip.acuk.mask]);
  i <- i+1
}

# sort indices of providers from most popular to least
idx <- sort(mta.serv.tot, index.return=TRUE, dec=TRUE)

# we use this to sort prov.nacuk, so that it won't be necessary to
# sort the results again later
mta.acuk <- mta.acuk[idx$ix]
mta.serv.tot <- idx$x

i <- 1
for (p in mta.acuk) {
  mta.serv.fe[i] <- nl(server[mta==p & serv.ip.acuk.mask
                              & type=="fe"]);
  mta.serv.he[i] <- nl(server[mta==p & serv.ip.acuk.mask
                              & type=="he"]);
  mta.serv.oth[i] <- nl(server[mta==p & serv.ip.acuk.mask
                               & (type=="none" | type=="unknown")]);
  i <- i+1
}

# create columns for output table
table.mta.serv.tot <- c(serv.ip.acuk.tot, serv.ip.acuk.tot - sum(mta.serv.tot),
                         sum(mta.serv.tot), mta.serv.tot)
table.mta.serv.fe <- c(serv.ip.acuk.fe, serv.ip.acuk.fe - sum(mta.serv.fe),
                        sum(mta.serv.fe), mta.serv.fe)
table.mta.serv.he <- c(serv.ip.acuk.he, serv.ip.acuk.he - sum(mta.serv.he),
                        sum(mta.serv.he), mta.serv.he)
table.mta.serv.oth <-c(serv.ip.acuk.oth, serv.ip.acuk.oth - sum(mta.serv.oth),
                        sum(mta.serv.oth), mta.serv.oth)

# create table
table.mta.serv <-
  data.frame(
             table.mta.serv.tot,
             pc(table.mta.serv.tot / table.mta.serv.tot[1]),
             table.mta.serv.fe,
             pc(table.mta.serv.fe / table.mta.serv.fe[1]),
             table.mta.serv.he,
             pc(table.mta.serv.he / table.mta.serv.he[1]),
             table.mta.serv.oth,
             pc(table.mta.serv.oth / table.mta.serv.oth[1]),
             row.names=c("Servers in ac.uk", "Running unknown MTAs",
               "Running known MTAs", mta.acuk))
names(table.mta.serv) <- c("Total", "(%)", "FE", "(%)", "HE", "(%)",
                       "Other", "(%)")

#table.mta.serv

# write table to CSV file
write("Mail Transfer Agents (MTAs) by ac.uk server",
      file="tables/table-mta-serv-acuk.csv")
write.csv(table.mta.serv,
          file="tables/table-mta-serv-acuk.csv", quote=FALSE,
          append=TRUE)

###########################################################
# MTA BY NON AC.UK SERVER TABLE
###########################################################

# number of providers
N <- length(mta.acuk)

# create vectors for counting institutions
mta.serv.tot <- array(0, N)

i <- 1
for (p in mta.acuk) {
  mta.serv.tot[i] <- nl(server[mta==p & serv.ip.nacuk.mask]);
  i <- i+1
}

# sort indices of providers from most popular to least
idx <- sort(mta.serv.tot, index.return=TRUE, dec=TRUE)

# we use this to sort prov.nacuk, so that it won't be necessary to
# sort the results again later
mta.acuk <- mta.acuk[idx$ix]
mta.serv.tot <- idx$x

# create columns for output table
table.mta.serv.tot <- c(serv.ip.nacuk.tot, serv.ip.nacuk.tot - sum(mta.serv.tot),
                         sum(mta.serv.tot), mta.serv.tot)

# create table
table.mta.serv <-
  data.frame(
             table.mta.serv.tot,
             pc(table.mta.serv.tot / table.mta.serv.tot[1]),
             row.names=c("Servers outside ac.uk", "Running unknown MTAs",
               "Running known MTAs", mta.acuk))
names(table.mta.serv) <- c("Total", "(%)")

#table.mta.serv

# write table to CSV file
write("Mail Transfer Agents (MTAs) by non ac.uk server",
      file="tables/table-mta-serv-nacuk.csv")
write.csv(table.mta.serv,
          file="tables/table-mta-serv-nacuk.csv", quote=FALSE,
          append=TRUE)

###########################################################
# FILTERS BY AC.UK SERVER TABLE
###########################################################

# number of providers
N <- length(filter.acuk)

# create vectors for counting institutions
filter.serv.tot <- array(0, N)
filter.serv.fe <- array(0, N)
filter.serv.he <- array(0, N)
filter.serv.oth <- array(0, N)

i <- 1
for (p in filter.acuk) {
  filter.serv.tot[i] <- nl(server[mta==p & serv.ip.acuk.mask]);
  i <- i+1
}

# sort indices of providers from most popular to least
idx <- sort(filter.serv.tot, index.return=TRUE, dec=TRUE)

# we use this to sort prov.nacuk, so that it won't be necessary to
# sort the results again later
filter.acuk <- filter.acuk[idx$ix]
filter.serv.tot <- idx$x

i <- 1
for (p in filter.acuk) {
  filter.serv.fe[i] <- nl(server[mta==p & serv.ip.acuk.mask
                                 & type=="fe"]);
  filter.serv.he[i] <- nl(server[mta==p & serv.ip.acuk.mask
                                 & type=="he"]);
  filter.serv.oth[i] <- nl(server[mta==p & serv.ip.acuk.mask
                                  & (type=="none" | type=="unknown")]);
  i <- i+1
}

# create columns for output table
table.filter.serv.tot <- c(serv.ip.acuk.tot,
                         sum(filter.serv.tot), filter.serv.tot)
table.filter.serv.fe <- c(serv.ip.acuk.fe,
                        sum(filter.serv.fe), filter.serv.fe)
table.filter.serv.he <- c(serv.ip.acuk.he,
                        sum(filter.serv.he), filter.serv.he)
table.filter.serv.oth <-c(serv.ip.acuk.oth,
                        sum(filter.serv.oth), filter.serv.oth)

# create table
table.filter.serv <-
  data.frame(
             table.filter.serv.tot,
             pc(table.filter.serv.tot / table.filter.serv.tot[1]),
             table.filter.serv.fe,
             pc(table.filter.serv.fe / table.filter.serv.fe[1]),
             table.filter.serv.he,
             pc(table.filter.serv.he / table.filter.serv.he[1]),
             table.filter.serv.oth,
             pc(table.filter.serv.oth / table.filter.serv.oth[1]),
             row.names=c("Servers in ac.uk",
               "Running filter", filter.acuk))
names(table.filter.serv) <- c("Total", "(%)", "FE", "(%)", "HE", "(%)",
                       "Other", "(%)")

#table.filter.serv

# write table to CSV file
write("Filters by ac.uk server",
      file="tables/table-filter-serv-acuk.csv")
write.csv(table.filter.serv,
          file="tables/table-filter-serv-acuk.csv", quote=FALSE,
          append=TRUE)

###########################################################
# FILTERS BY NON AC.UK SERVER TABLE
###########################################################

# number of providers
N <- length(filter.acuk)

# create vectors for counting institutions
filter.serv.tot <- array(0, N)

i <- 1
for (p in filter.acuk) {
  filter.serv.tot[i] <- nl(server[mta==p & serv.ip.nacuk.mask]);
  i <- i+1
}

# sort indices of providers from most popular to least
idx <- sort(filter.serv.tot, index.return=TRUE, dec=TRUE)

# we use this to sort prov.nacuk, so that it won't be necessary to
# sort the results again later
filter.acuk <- filter.acuk[idx$ix]
filter.serv.tot <- idx$x

# create columns for output table
table.filter.serv.tot <- c(serv.ip.nacuk.tot,
                         sum(filter.serv.tot), filter.serv.tot)

# create table
table.filter.serv <-
  data.frame(
             table.filter.serv.tot,
             pc(table.filter.serv.tot / table.filter.serv.tot[1]),
             row.names=c("Servers outside ac.uk",
               "Running filter", filter.acuk))
names(table.filter.serv) <- c("Total", "(%)")

#table.filter.serv

# write table to CSV file
write("Filters by non ac.uk server",
      file="tables/table-filter-serv-nacuk.csv")
write.csv(table.filter.serv,
          file="tables/table-filter-serv-nacuk.csv", quote=FALSE,
          append=TRUE)
